apiVersion: 1
groups:
    - orgId: 1
      name: IP
      folder: DeepThought
      interval: 5m
      rules:
        - uid: e0401517-8318-4e84-a8e6-ed7155fc68c9
          title: IP addresses differ
          condition: C
          data:
            - refId: A
              relativeTimeRange:
                from: 600
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                disableTextWrap: false
                editorMode: builder
                expr: count by(address) (ip_address)
                fullMetaSearch: false
                includeNullMetadata: true
                instant: true
                intervalMs: 1000
                legendFormat: __auto
                maxDataPoints: 43200
                range: false
                refId: A
                useBackend: false
            - refId: B
              relativeTimeRange:
                from: 600
                to: 0
              datasourceUid: __expr__
              model:
                conditions:
                    - evaluator:
                        params: []
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                            - B
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: __expr__
                expression: A
                intervalMs: 1000
                maxDataPoints: 43200
                reducer: last
                refId: B
                type: reduce
            - refId: C
              relativeTimeRange:
                from: 600
                to: 0
              datasourceUid: __expr__
              model:
                conditions:
                    - evaluator:
                        params:
                            - 2
                        type: lt
                      operator:
                        type: and
                      query:
                        params:
                            - C
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: __expr__
                expression: B
                intervalMs: 1000
                maxDataPoints: 43200
                refId: C
                type: threshold
          noDataState: NoData
          execErrState: Error
          for: 5m
          isPaused: false
    - orgId: 1
      name: Media
      folder: DeepThought
      interval: 1m
      rules:
        - uid: e896688e-54b2-4d69-b10f-c79a17fbef8c
          title: Media errors
          condition: C
          data:
            - refId: A
              relativeTimeRange:
                from: 600
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                editorMode: code
                expr: node_btrfs_device_errors_total{device="J"}
                hide: false
                intervalMs: 1000
                legendFormat: __auto
                maxDataPoints: 43200
                range: true
                refId: A
            - refId: B
              relativeTimeRange:
                from: 600
                to: 0
              datasourceUid: __expr__
              model:
                conditions:
                    - evaluator:
                        params: []
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                            - B
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: __expr__
                expression: A
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                reducer: last
                refId: B
                type: reduce
            - refId: C
              relativeTimeRange:
                from: 600
                to: 0
              datasourceUid: __expr__
              model:
                conditions:
                    - evaluator:
                        params:
                            - 0
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                            - C
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: __expr__
                expression: B
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                refId: C
                type: threshold
          noDataState: OK
          execErrState: OK
          for: 5m
          isPaused: false
    - orgId: 1
      name: CPU temperature too high
      folder: DeepThought
      interval: 1m
      rules:
        - uid: xsAtJ6ynz
          title: CPU temperature too high
          condition: F
          data:
            - refId: A
              relativeTimeRange:
                from: 86400
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                datasource:
                    type: prometheus
                    uid: PBFA97CFB590B2093
                expr: node_hwmon_temp_celsius{instance="localhost:9100",job="node_exporter"}
                format: time_series
                interval: ""
                intervalFactor: 1
                intervalMs: 15000
                legendFormat: '{{ chip }} {{ sensor }} temp'
                maxDataPoints: 43200
                refId: A
                step: 240
            - refId: F
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 75
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                            - A
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: "-100"
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                refId: F
                type: classic_conditions
          dashboardUid: rYdddlPWk
          panelId: 158
          noDataState: NoData
          execErrState: Alerting
          for: 5m
          annotations:
            __dashboardUid__: rYdddlPWk
            __panelId__: "158"
          isPaused: false
    - orgId: 1
      name: HDD
      folder: DeepThought
      interval: 1m
      rules:
        - uid: LpX5Ve44z
          title: Below 10GiB space available on root
          condition: B
          data:
            - refId: A
              relativeTimeRange:
                from: 86400
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                datasource:
                    type: prometheus
                    uid: PBFA97CFB590B2093
                editorMode: code
                expr: node_filesystem_avail_bytes{instance="localhost:9100",job="node_exporter",device!~'rootfs', mountpoint="/"}
                format: time_series
                hide: false
                interval: ""
                intervalFactor: 1
                intervalMs: 15000
                legendFormat: '{{mountpoint}} - Available'
                maxDataPoints: 43200
                metric: ""
                range: true
                refId: A
                step: 240
            - refId: B
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 1.073741824e+10
                            - 0
                        type: lt
                      operator:
                        type: and
                      query:
                        params:
                            - A
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    name: Expression
                    type: __expr__
                    uid: __expr__
                expression: A
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                refId: B
                type: classic_conditions
          dashboardUid: rYdddlPWk
          panelId: 43
          noDataState: NoData
          execErrState: Alerting
          for: 5m
          annotations:
            __dashboardUid__: rYdddlPWk
            __panelId__: "43"
          isPaused: false
    - orgId: 1
      name: Power supply
      folder: DeepThought
      interval: 30s
      rules:
        - uid: aglE8VB7z
          title: No Power Supply
          condition: B
          data:
            - refId: A
              relativeTimeRange:
                from: 86400
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                datasource:
                    type: prometheus
                    uid: PBFA97CFB590B2093
                expr: node_power_supply_online{instance="localhost:9100",job="node_exporter"}
                format: time_series
                hide: false
                interval: ""
                intervalFactor: 1
                intervalMs: 15000
                legendFormat: '{{ power_supply }} online'
                maxDataPoints: 43200
                refId: A
                step: 240
            - refId: B
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 1
                        type: lt
                      operator:
                        type: and
                      query:
                        params:
                            - A
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: "-100"
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                refId: B
                type: classic_conditions
          dashboardUid: rYdddlPWk
          panelId: 302
          noDataState: OK
          execErrState: OK
          for: 1m
          annotations:
            __dashboardUid__: rYdddlPWk
            __panelId__: "302"
          isPaused: false
    - orgId: 1
      name: process
      folder: DeepThought
      interval: 1m
      rules:
        - uid: 3WIFI0cVz
          title: dnscrypt not running in 2 instances
          condition: B
          data:
            - refId: A
              relativeTimeRange:
                from: 21600
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                datasource:
                    type: prometheus
                    uid: PBFA97CFB590B2093
                editorMode: builder
                expr: namedprocess_namegroup_num_procs{groupname="dnscrypt-proxy"}
                interval: ""
                intervalMs: 15000
                legendFormat: '{{groupname}}'
                maxDataPoints: 43200
                range: true
                refId: A
            - refId: B
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 2
                        type: lt
                      operator:
                        type: and
                      query:
                        params:
                            - A
                      reducer:
                        params: []
                        type: last
                      type: query
                    - evaluator:
                        params:
                            - 0
                            - 0
                        type: no_value
                      operator:
                        type: or
                      query:
                        params:
                            - A
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: "-100"
                expression: A
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                refId: B
                type: classic_conditions
          dashboardUid: CrRW7Vf7k
          panelId: 16
          noDataState: NoData
          execErrState: Error
          for: 5m
          annotations:
            __dashboardUid__: CrRW7Vf7k
            __panelId__: "16"
          labels:
            hush: backup
          isPaused: false
        - uid: rsY-R8c4z
          title: Process not running at all
          condition: C
          data:
            - refId: A
              relativeTimeRange:
                from: 21600
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                datasource:
                    type: prometheus
                    uid: PBFA97CFB590B2093
                editorMode: builder
                exemplar: false
                expr: namedprocess_namegroup_num_procs{groupname=~"amuse|doh-server|miniflux|bimba_server|smithy|glider|honk|website|crond|grafana-server|hydroxide|nginx|prometheus|radicale|toymaker.sh|transmission-da|asgard_serve|dendrite"}
                format: time_series
                instant: false
                interval: ""
                intervalMs: 15000
                legendFormat: '{{groupname}}'
                maxDataPoints: 43200
                range: true
                refId: A
            - refId: B
              relativeTimeRange:
                from: 21600
                to: 0
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 0
                            - 0
                        type: gt
                      operator:
                        type: and
                      query:
                        params: []
                      reducer:
                        params: []
                        type: avg
                      type: query
                datasource:
                    name: Expression
                    type: __expr__
                    uid: __expr__
                expression: A
                intervalMs: 1000
                maxDataPoints: 43200
                reducer: last
                refId: B
                type: reduce
            - refId: C
              relativeTimeRange:
                from: 21600
                to: 0
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 1
                            - 0
                        type: lt
                      operator:
                        type: and
                      query:
                        params: []
                      reducer:
                        params: []
                        type: avg
                      type: query
                datasource:
                    name: Expression
                    type: __expr__
                    uid: __expr__
                expression: B
                intervalMs: 1000
                maxDataPoints: 43200
                refId: C
                type: threshold
          dashboardUid: CrRW7Vf7k
          panelId: 18
          noDataState: NoData
          execErrState: Error
          for: 5m
          annotations:
            __dashboardUid__: CrRW7Vf7k
            __panelId__: "18"
            summary: '{{ $labels.groupname }} not running'
          labels:
            hush: backup
          isPaused: false
        - uid: MCzwgl54z
          title: asgard minion run in too many instance
          condition: B
          data:
            - refId: A
              relativeTimeRange:
                from: 21600
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                datasource:
                    type: prometheus
                    uid: PBFA97CFB590B2093
                editorMode: builder
                expr: namedprocess_namegroup_num_procs{groupname=~"asgard_tyr|asgard_hermodr"}
                interval: ""
                intervalMs: 15000
                legendFormat: '{{groupname}}'
                maxDataPoints: 43200
                range: true
                refId: A
            - refId: B
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 0
                            - 0
                        type: gt
                      operator:
                        type: and
                      query:
                        params: []
                      reducer:
                        params: []
                        type: avg
                      type: query
                datasource:
                    name: Expression
                    type: __expr__
                    uid: __expr__
                expression: A
                intervalMs: 1000
                maxDataPoints: 43200
                reducer: last
                refId: B
                type: reduce
            - refId: C
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 1
                            - 0
                        type: gt
                      operator:
                        type: and
                      query:
                        params: []
                      reducer:
                        params: []
                        type: avg
                      type: query
                datasource:
                    name: Expression
                    type: __expr__
                    uid: __expr__
                expression: B
                intervalMs: 1000
                maxDataPoints: 43200
                refId: C
                type: threshold
          dashboardUid: CrRW7Vf7k
          panelId: 20
          noDataState: OK
          execErrState: Alerting
          for: 5m
          annotations:
            __dashboardUid__: CrRW7Vf7k
            __panelId__: "20"
            summary: asgard minion {{ $labels.groupname }} run in too many instances
          isPaused: false
        - uid: 8dJxb654z
          title: Container not running
          condition: C
          data:
            - refId: A
              relativeTimeRange:
                from: 21600
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                datasource:
                    type: prometheus
                    uid: PBFA97CFB590B2093
                editorMode: code
                expr: sum by(name) ((podman_container_info{name!~".*-infra"}) * on(id) group_right(name) podman_container_state)
                interval: ""
                intervalMs: 15000
                legendFormat: __auto
                maxDataPoints: 43200
                range: true
                refId: A
            - refId: B
              relativeTimeRange:
                from: 21600
                to: 0
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params: []
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                            - B
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: "-100"
                expression: A
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                reducer: last
                refId: B
                type: reduce
            - refId: C
              relativeTimeRange:
                from: 21600
                to: 0
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 2
                            - 2
                        type: outside_range
                      operator:
                        type: and
                      query:
                        params:
                            - C
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: "-100"
                expression: B
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                refId: C
                type: threshold
          dashboardUid: CrRW7Vf7k
          panelId: 22
          noDataState: NoData
          execErrState: Error
          for: 5m
          annotations:
            __dashboardUid__: CrRW7Vf7k
            __panelId__: "22"
            summary: '{{ $labels.name }} not running'
          labels:
            hush: backup
          isPaused: false
    - orgId: 1
      name: wireguard
      folder: DeepThought
      interval: 1m
      rules:
        - uid: 35ASf5H4k
          title: Peer Last Handshake above 5m
          condition: B
          data:
            - refId: A
              relativeTimeRange:
                from: 3600
                to: 0
              datasourceUid: PBFA97CFB590B2093
              model:
                datasource:
                    type: prometheus
                    uid: PBFA97CFB590B2093
                editorMode: code
                expr: time()-(wireguard_latest_handshake_seconds)
                format: time_series
                interval: ""
                intervalMs: 1000
                legendFormat: '{{public_key}}'
                maxDataPoints: 43200
                range: true
                refId: A
            - refId: B
              datasourceUid: "-100"
              model:
                conditions:
                    - evaluator:
                        params:
                            - 300
                        type: gt
                      operator:
                        type: and
                      query:
                        params:
                            - A
                      reducer:
                        params: []
                        type: last
                      type: query
                datasource:
                    type: __expr__
                    uid: "-100"
                expression: A
                hide: false
                intervalMs: 1000
                maxDataPoints: 43200
                refId: B
                type: classic_conditions
          dashboardUid: romYcXqZk
          panelId: 6
          noDataState: NoData
          execErrState: Alerting
          for: 5m
          annotations:
            __dashboardUid__: romYcXqZk
            __panelId__: "6"
          isPaused: false
